#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: main.py
@time: 2017/12/20 14:59
"""
import time

import gevent.pool
from gevent import monkey

from crawler.jiangsu.company_page_crawler import CompanyPageCrawler

monkey.patch_all()

import click

from logger import Logger

# 日志模块
global_log = Logger('old_jiangsu_crawl.log').get_logger()

is_running = True


# 休眠
def check_sleep(times):
    count = 0
    while count < times and is_running:
        count += 1
        time.sleep(1)


# 获得模块日志名称
def get_log_name(province):
    return 'crawl_{}.log'.format(province)


# 列表页抓取入口
def crawl_runner(seed_table, thread_num, page_num, store_tube):
    global_log.info("列表页抓取启动: seed_table = {}".format(seed_table))

    # 生成模块日志
    log_name = get_log_name(seed_table)
    crawler = CompanyPageCrawler(store_tube, seed_table, Logger(log_name).get_logger())

    # 初始化线程池
    pool = gevent.pool.Pool(thread_num)
    global_log.info('启用协程...')
    global_log.info('当前开启协程数目: thread_num = {}'.format(thread_num))

    result_list = []

    for page in xrange(1, page_num + 1):
        result_list.append(pool.apply_async(crawler.crawl_process, args=(page,)))
        if len(result_list) >= 10000:
            for result in result_list:
                result.get()
            del result_list[:]

    if len(result_list) > 0:
        for result in result_list:
            result.get()

    # 等待线程结束
    pool.join()


@click.command()
@click.option('--seed_table',
              default='offline_jiangsu_list',
              help='旧版浙江工商种子表')
@click.option('--thread_num',
              default=1,
              type=int,
              help='需要开启的并发数目')
@click.option('--page_num',
              default=50382,
              type=int,
              help='需要开启的并发数目')
@click.option('--store_tube',
              default='gsxt_store',
              help='存储消息队列')
def main(seed_table, thread_num, page_num, store_tube):
    try:
        # 运行抓取
        crawl_runner(seed_table, thread_num, page_num=page_num, store_tube=store_tube)
    except Exception as e:
        global_log.error("程序退出异常: ")
        global_log.exception(e)


if __name__ == '__main__':
    main()
